
package com.djhu.newscollector.crawel.net;

import com.djhu.newscollector.core.job.BaseJob;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;


import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Page是爬取过程中，内存中保存网页爬取信息的一个容器，Page只在内存中存 放，用于保存一些网页信息，方便用户进行自定义网页解析之类的操作。
 *
 * @author hu
 */
public class Page implements BaseJob{

    public static final Logger LOG = LogManager.getLogger(Page.class);

    private HttpRequest request = null;
    private HttpResponse response = null;

    private String contentType;
    private Exception exception = null;

    private Document doc = null;

    private String charset = null;

    private Object obj = null;
    private int excutetimes = 0;
    public Page() {
    }

    /**
     * 判断当前Page的URL是否和输入正则匹配
     *
     * @param urlRegex
     * @return
     */
    public boolean matchUrl(String urlRegex) {
        return Pattern.matches(urlRegex, url());
    }

    /**
     * 判断当前Page的Http响应头的Content-Type是否符合正则
     *
     * @param contentTypeRegex
     * @return
     */
    public boolean matchContentType(String contentTypeRegex) {
        if (contentTypeRegex == null) {
            return contentType() == null;
        }
        return Pattern.matches(contentTypeRegex, contentType());
    }



    /**
     * 获取网页中满足指定css选择器的所有元素的指定属性的集合
     * 例如通过attrs("img[src]","abs:src")可获取网页中所有图片的链接
     *
     * @param cssSelector
     * @param attrName
     * @return
     */
    public ArrayList<String> attrs(String cssSelector, String attrName) {
        ArrayList<String> result = new ArrayList<String>();
        Elements eles = select(cssSelector);
        for (Element ele : eles) {
            if (ele.hasAttr(attrName)) {
                result.add(ele.attr(attrName));
            }
        }
        return result;
    }

    /**
     * 获取网页中满足指定css选择器的所有元素的指定属性的集合
     * 例如通过attr("img[src]","abs:src")可获取网页中第一个图片的链接
     *
     * @param cssSelector
     * @param attrName
     * @return
     */
    public String attr(String cssSelector, String attrName) {
        return select(cssSelector).attr(attrName);
    }


//    public Links links(boolean parseImg) {
//        Links links = new Links().addFromElement(doc(),parseImg);
//        return links;
//    }
//
//    public Links links() {
//        return links(false);
//    }


    
        /**
     * 获取满足选择器的元素中的链接 选择器cssSelector必须定位到具体的超链接 例如我们想抽取id为content的div中的所有超链接，这里
     * 就要将cssSelector定义为div[id=content] a
     *
     * @param cssSelector
     * @return
     */
//    public Links links(String cssSelector, boolean parseSrc) {
//        Links links = new Links().addBySelector(doc(), cssSelector,parseSrc);
//        return links;
//    }
//    public Links links(String cssSelector) {
//        return links(cssSelector,false);
//    }




//    public Links regexLinks(RegexRule regexRule, boolean parseSrc) {
//        return new Links().addByRegex(doc(), regexRule, parseSrc);
//    }
//    public Links regexLinks(String regex, boolean parseSrc){
//        return new Links().addByRegex(doc(),regex,parseSrc);
//    }
//
//    public Links regexLinks(RegexRule regexRule) {
//        return regexLinks(regexRule, false);
//    }
//    public Links regexLinks(String regex){
//        return regexLinks(regex,false);
//    }


    public ArrayList<String> selectTextList(String cssSelector){
        ArrayList<String> result = new ArrayList<String>();
        Elements eles = select(cssSelector);
        for(Element ele:eles){
            result.add(ele.text());
        }
        return result;
    }

//    public String selectText(String cssSelector, int index){
//        return ListUtils.getByIndex(selectTextList(cssSelector),index);
//    }
    public String selectText(String cssSelector) {
        return select(cssSelector).first().text();
    }

    public ArrayList<Integer> selectIntList(String cssSelector){
        ArrayList<Integer> result = new ArrayList<Integer>();
        for(String text:selectTextList(cssSelector)){
            result.add(Integer.valueOf(text.trim()));
        }
        return result;
    }


//    public int selectInt(String cssSelector){
//        return selectInt(cssSelector,0);
//    }

    public ArrayList<Double> selectDoubleList(String cssSelector){
        ArrayList<Double> result = new ArrayList<Double>();
        for(String text:selectTextList(cssSelector)){
            result.add(Double.valueOf(text.trim()));
        }
        return result;
    }



    public ArrayList<Long> selectLongList(String cssSelector){
        ArrayList<Long> result = new ArrayList<Long>();
        for(String text:selectTextList(cssSelector)){
            result.add(Long.valueOf(text.trim()));
        }
        return result;
    }



    public Elements select(String cssSelector) {
        return this.doc().select(cssSelector);
    }

    public Element select(String cssSelector, int index) {
        Elements eles = select(cssSelector);
        int realIndex = index;
        if (index < 0) {
            realIndex = eles.size() + index;
        }
        return eles.get(realIndex);
    }

    public String regex(String regex, int group, String defaultResult) {
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(html());
        if (matcher.find()) {
            return matcher.group(group);
        } else {
            return defaultResult;
        }
    }

    public String regex(String regex, int group) {
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(html());
        matcher.find();
        return matcher.group(group);
    }

    public String regexAndFormat(String regex, String format){
        Pattern pattern = Pattern.compile(regex);
        Matcher matcher = pattern.matcher(html());
        matcher.find();
        String[] strs = new String[matcher.groupCount()];
        for(int i=0;i<matcher.groupCount();i++){
            strs[i] = matcher.group(i+1);
        }
        return String.format(format, strs);
    }

    public String regex(String regex, String defaultResult) {
        return regex(regex, 0, defaultResult);
    }

    public String regex(String regex) {
        return regex(regex, 0);
    }


    /**
     * 返回网页/文件的内容
     *
     * @return 网页/文件的内容
     */
    public byte[] content() {
        return response.getBytes();
    }

    /**
     * 返回网页的url
     *
     * @return 网页的url
     */
    public String url() {
        return this.response.getRequest().getUrl().getUrl();
    }


    /**
     * 返回网页的源码字符串
     *
     * @return 网页的源码字符串
     */
    public String html() {

        return response.getHtml();
    }



    public String contentType() {
        return contentType;
    }

    /**
     * 返回网页解析后的DOM树(Jsoup的Document对象) 已废弃，使用doc()方法代替
     *
     * @return 网页解析后的DOM树
     */
    public Document doc() {
        if (doc != null) {
            return doc;
        }
        try {

            this.doc = Jsoup.parse(html(), url());
            return doc;
        } catch (Exception ex) {
            LOG.info("Exception", ex);
            return null;
        }
    }

    /**
     * 设置网页解析后的DOM树(Jsoup的Document对象)
     *
     * @param doc 网页解析后的DOM树
     */
    public void doc(Document doc){
        this.doc = doc;
    }



    public Exception getException() {
        return exception;
    }

    public void setException(Exception exception) {
        this.exception = exception;
    }


//    public void meta(HashMap<String, String> metaData) {
//        this.crawlDatum.meta(metaData);
//    }
//
//    public void meta(String key, String value) {
//        this.crawlDatum.meta(key, value);
//    }




    public String charset() {
        if (charset == null) {
            charset = CharsetDetector.guessEncoding(content());
        }
        return charset;
    }

    public void charset(String charset) {
        this.charset = charset;
    }




    public int code() {
        return response.getCode();
    }


    public <T> T obj() {
        return (T)obj;
    }

    public void obj(Object obj) {
        this.obj = obj;
    }


    public HttpRequest getRequest() {
        return response.getRequest();
    }


    public HttpResponse getResponse() {
        return response;
    }

    public void setResponse(HttpResponse response) {
        this.response = response;
    }

    @Override
    public String id() {
        return request.getUrl().getUrl();
    }

    @Override
    public String name() {
        return "PAGE";
    }

    @Override
    public Integer priority() {
        return 1;
    }

    public Integer excuteTimes() {
        return excutetimes;
    }

    public Integer excuteTimes(int excuteTimes) {
        this.excutetimes = excuteTimes;
        return this.excutetimes;
    }

    public Set<String> getLinks() {
        Elements links = doc().select("a[href]");
        Set<String> urls = new HashSet<>();
        for (Element element:links) {
            String link = element.attr("abs:href");
            if (StringUtils.isEmpty(link)) {
                continue;
            }
            urls.add(link);
        }
        return urls;
    }
}
